{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import time\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.model_selection import GridSearchCV,StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"RentListingInquries_FE_train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#用来判断某列是否有缺失值\n",
    "#print(train.isnull().any())\n",
    "\n",
    "y_train = train['interest_level']\n",
    "x_train = train.drop(['interest_level'],axis=1)\n",
    "x_train = np.array(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5,shuffle=True,random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "\n",
    "parms = dict(max_depth=max_depth,min_child_weight=min_child_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#确定树的最大深度和权重, 这是一个三分类的问题 [2 1 0] 为高、中、低三类\n",
    "xgb1 = XGBClassifier(\n",
    "    learning_rate=0.1,\n",
    "    n_estimators=205,\n",
    "    gamma=0,\n",
    "    subsample=0.3,\n",
    "    colsample_bytree=0.8,\n",
    "    colsample_bylevel=0.7,\n",
    "    objective='multi:softprob',\n",
    "    seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time.struct_time(tm_year=2018, tm_mon=5, tm_mday=27, tm_hour=22, tm_min=31, tm_sec=32, tm_wday=6, tm_yday=147, tm_isdst=0)\n",
      "time.struct_time(tm_year=2018, tm_mon=5, tm_mday=28, tm_hour=1, tm_min=18, tm_sec=39, tm_wday=0, tm_yday=148, tm_isdst=0)\n",
      "{'mean_fit_time': array([  90.19383245,   91.09554024,   89.44234691,  146.8524756 ,\n",
      "        138.70003743,  135.33213544,  189.15064554,  188.39939075,\n",
      "        195.82841024,  252.71374512,  208.5274672 ,  207.3783021 ]), 'std_fit_time': array([  1.83013779,   2.49328716,   3.51565261,   9.15994173,\n",
      "         5.81650647,   0.84239302,   1.69728867,   0.68022336,\n",
      "        12.92411187,  31.07169697,   0.49653896,   0.95038561]), 'mean_score_time': array([ 0.22287412,  0.2470571 ,  0.26384778,  0.39390798,  0.36579361,\n",
      "        0.37003736,  0.52905135,  0.54617705,  0.5617126 ,  1.16176205,\n",
      "        0.71811986,  0.69839673]), 'std_score_time': array([ 0.01557816,  0.01475763,  0.06656174,  0.05648848,  0.04169449,\n",
      "        0.05414282,  0.03639731,  0.04975394,  0.09506672,  0.32230221,\n",
      "        0.0109069 ,  0.0377959 ]), 'param_max_depth': masked_array(data = [3 3 3 5 5 5 7 7 7 9 9 9],\n",
      "             mask = [False False False False False False False False False False False False],\n",
      "       fill_value = ?)\n",
      ", 'param_min_child_weight': masked_array(data = [1 3 5 1 3 5 1 3 5 1 3 5],\n",
      "             mask = [False False False False False False False False False False False False],\n",
      "       fill_value = ?)\n",
      ", 'params': [{'max_depth': 3, 'min_child_weight': 1}, {'max_depth': 3, 'min_child_weight': 3}, {'max_depth': 3, 'min_child_weight': 5}, {'max_depth': 5, 'min_child_weight': 1}, {'max_depth': 5, 'min_child_weight': 3}, {'max_depth': 5, 'min_child_weight': 5}, {'max_depth': 7, 'min_child_weight': 1}, {'max_depth': 7, 'min_child_weight': 3}, {'max_depth': 7, 'min_child_weight': 5}, {'max_depth': 9, 'min_child_weight': 1}, {'max_depth': 9, 'min_child_weight': 3}, {'max_depth': 9, 'min_child_weight': 5}], 'split0_test_score': array([-0.6030777 , -0.60258986, -0.60259565, -0.58848971, -0.58850584,\n",
      "       -0.58926743, -0.58769648, -0.58614777, -0.58732167, -0.60143251,\n",
      "       -0.5945876 , -0.59596132]), 'split1_test_score': array([-0.6060509 , -0.60565587, -0.60601663, -0.59472096, -0.59445759,\n",
      "       -0.59482708, -0.59420128, -0.5936818 , -0.59132576, -0.60423491,\n",
      "       -0.59929554, -0.59625964]), 'split2_test_score': array([-0.60480271, -0.60447268, -0.60454278, -0.59070815, -0.59047944,\n",
      "       -0.59134413, -0.58998387, -0.59068856, -0.59085982, -0.60644614,\n",
      "       -0.59655114, -0.59548073]), 'split3_test_score': array([-0.60034884, -0.60041598, -0.5996308 , -0.58653608, -0.5854211 ,\n",
      "       -0.58606431, -0.58525615, -0.58692615, -0.58524324, -0.59497796,\n",
      "       -0.58844484, -0.58969095]), 'split4_test_score': array([-0.60398166, -0.60336461, -0.6040145 , -0.59030165, -0.59092055,\n",
      "       -0.59182011, -0.59121642, -0.59168478, -0.58877362, -0.60349817,\n",
      "       -0.60218743, -0.60048148]), 'mean_test_score': array([-0.60365234, -0.6032998 , -0.60336003, -0.5901513 , -0.58995685,\n",
      "       -0.59066454, -0.58967075, -0.5898257 , -0.58870482, -0.60211785,\n",
      "       -0.59621295, -0.59557453]), 'std_test_score': array([ 0.00191986,  0.00177495,  0.00216229,  0.00272204,  0.00297117,\n",
      "        0.00290655,  0.00304635,  0.00286378,  0.00225456,  0.00391339,\n",
      "        0.00465386,  0.00344438]), 'rank_test_score': array([12, 10, 11,  5,  4,  6,  2,  3,  1,  9,  8,  7]), 'split0_train_score': array([-0.58233339, -0.58289852, -0.58339039, -0.51921432, -0.52369682,\n",
      "       -0.52742133, -0.4202023 , -0.44003098, -0.45069069, -0.3014208 ,\n",
      "       -0.34627544, -0.37098868]), 'split1_train_score': array([-0.58187358, -0.58281784, -0.58249838, -0.51811428, -0.52322662,\n",
      "       -0.52681759, -0.42075111, -0.4406268 , -0.45217547, -0.30253785,\n",
      "       -0.34508817, -0.37220192]), 'split2_train_score': array([-0.58148534, -0.58209015, -0.58229845, -0.51754952, -0.52309605,\n",
      "       -0.52665072, -0.42049075, -0.43865653, -0.45225078, -0.29882152,\n",
      "       -0.34368171, -0.3695062 ]), 'split3_train_score': array([-0.58235592, -0.58313556, -0.583609  , -0.52023968, -0.52544135,\n",
      "       -0.52824205, -0.42104939, -0.43916257, -0.45188904, -0.29936367,\n",
      "       -0.34271934, -0.36891507]), 'split4_train_score': array([-0.5821522 , -0.58276371, -0.58271148, -0.51724869, -0.52354103,\n",
      "       -0.52711927, -0.42157503, -0.44119832, -0.45128606, -0.30439263,\n",
      "       -0.3451337 , -0.37187474]), 'mean_train_score': array([-0.58204008, -0.58274116, -0.58290154, -0.5184733 , -0.52380038,\n",
      "       -0.52725019, -0.42081372, -0.43993504, -0.45165841, -0.30130729,\n",
      "       -0.34457967, -0.37069732]), 'std_train_score': array([ 0.00032674,  0.00034944,  0.00051027,  0.00110916,  0.00084806,\n",
      "        0.00056156,  0.0004727 ,  0.00092897,  0.00059103,  0.00204955,\n",
      "        0.00124164,  0.00129075])}\n",
      "######################################################################\n",
      "Best: -0.588705 using {'max_depth': 7, 'min_child_weight': 5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text(0,0.5,'Log Loss')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_sea = GridSearchCV(xgb1,param_grid=parms,scoring=\"neg_log_loss\",cv=kfold)\n",
    "\n",
    "print(time.localtime(time.time()))\n",
    "grid_sea.fit(x_train,y_train)\n",
    "print(time.localtime(time.time()))\n",
    "\n",
    "# print(grid_sea.grid_scores_,grid_sea.best_params_,grid_sea.best_score_)\n",
    "print(grid_sea.cv_results_)\n",
    "print(\"######################################################################\")\n",
    "print(\"Best: %f using %s\" % (grid_sea.best_score_, grid_sea.best_params_))\n",
    "\n",
    "test_means = grid_sea.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid_sea.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid_sea.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid_sea.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "pd.DataFrame(grid_sea.cv_results_).to_csv('house_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    plt.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "\n",
    "plt.legend()\n",
    "plt.xlabel( 'max_depth' )\n",
    "plt.ylabel( 'Log Loss' )"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 运行时间约3小时 。 结果： {'max_depth': 7, 'min_child_weight': 5}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
